{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb\n",
    "from sklearn.preprocessing import LabelEncoder,OneHotEncoder\n",
    "from sklearn.model_selection import KFold,StratifiedKFold\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score,roc_auc_score,f1_score,recall_score\n",
    "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer,HashingVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD,SparsePCA\n",
    "import gc\n",
    "import time\n",
    "import os\n",
    "import sys\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "train = pd.read_csv(\"../data2/data/age_train.csv\",names=['uid','age_group']).sort_values(by=['uid'])\n",
    "test = pd.read_csv(\"../data2/data/age_test.csv\",names=['uid']).sort_values(by=['uid'])\n",
    "info = pd.read_csv(\"../data2/data/app_info.csv\",names=['appid','category'])\n",
    "active = pd.read_csv(\"../data2/data/user_app_actived.csv\",names=['uid','appid']).sort_values(by=['uid'])\n",
    "usage = pd.read_pickle(\"../data2/user_app_usage.pickle\")#,names=['uid','appid','duration','times','use_date'],parse_dates=['use_date'])\n",
    "user_basic_info = pd.read_csv(\"../data2/data/user_basic_info.csv\",names=['uid','gender','city','prodname','ramcapacity','ramleftration','romcapacity','romleftration','color','fontsize','ct','carrier','os']).sort_values(by=['uid'])\n",
    "behavior_info = pd.read_csv(\"../data2/data/user_behavior_info.csv\",names=['uid','boottimes','a','b','c','d','e','f','g']).sort_values(by=['uid'])\n",
    "# (train.shape,test.shape),(info.shape,active.shape,usage.shape,user_basic_info.shape,behavior_info.shape)\n",
    "\n",
    "all_data = train.append(test).reset_index(drop=True)\n",
    "all_data.head()\n",
    "\n",
    "active['appid'] = active['appid'].map(lambda x:x.split('#'))\n",
    "active['app_len'] = active['appid'].map(lambda x:len(x))\n",
    "\n",
    "def get_category(x):\n",
    "    col = []\n",
    "    no_col = 0\n",
    "    for i in x:\n",
    "        try:\n",
    "            col.append(hash_dict[i])\n",
    "        except:\n",
    "            no_col+=1\n",
    "    return col,no_col\n",
    "\n",
    "hash_dict = dict(info.values)\n",
    "active['category'] = active['appid'].map(lambda x:get_category(x))\n",
    "active['category_nan']  = active['category'].map(lambda x:x[1])\n",
    "active['category']  = active['category'].map(lambda x:x[0])\n",
    "active['category_len'] = active['category'].map(lambda x:len(x))\n",
    "active['category_nunique'] = active['category'].map(lambda x:len(set(x)))\n",
    "active['category_ratio'] = active['category_nunique']/active['category_len']\n",
    "del active['category']\n",
    "\n",
    "all_data = all_data.sort_values(by=['uid']).reset_index(drop=True)\n",
    "all_data = all_data.merge(user_basic_info,how='left',on=['uid'])\n",
    "all_data = all_data.merge(behavior_info,how='left',on=['uid'])\n",
    "all_data = all_data.merge(active[active['app_len']<=150],how='left',on=['uid'])\n",
    "lj = active[active['app_len']>150]\n",
    "active = pd.read_csv(\"../data2/data/user_app_actived.csv\",names=['uid','appid']).sort_values(by=['uid'])\n",
    "active.reset_index(drop=True,inplace=True)\n",
    "\n",
    "active.loc[active['uid'].isin(lj['uid'].unique()),'appid'] = '#a00101827'\n",
    "all_data = all_data.merge(active.rename(columns={'appid' : 'multi_appid'}),how='left',on='uid')\n",
    "\n",
    "def split(x):\n",
    "    key_ans = x.split('#')\n",
    "    for key in key_ans:\n",
    "        if key not in key2index:\n",
    "            # Notice : data value 0 is a special \"padding\",so we do not use 0 to encode valid feature for sequence data\n",
    "            key2index[key] = len(key2index) + 1\n",
    "    return list(map(lambda x: key2index[x], key_ans))\n",
    "\n",
    "all_data['multi_appid'] = all_data['multi_appid'].astype('str')\n",
    "from tensorflow.python.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# 多值特征处理\n",
    "from deepctr.models import *\n",
    "from deepctr.datas import SparseFeat, VarLenSparseFeat,get_fixlen_feature_names,get_varlen_feature_names,DenseFeat\n",
    "key2index = {}\n",
    "app_list = list(map(split, all_data['multi_appid'].values))\n",
    "app_length = np.array(list(map(len, app_list)))\n",
    "app_key = key2index.copy()\n",
    "\n",
    "max_len_app = max(app_length)\n",
    "app_list = pad_sequences(app_list, maxlen=max_len_app, padding='post', )\n",
    "print(max_len_app)\n",
    "\n",
    "uid_seq = pd.read_pickle(\"usage_uid_appid_seq.pickle\")\n",
    "uid_seq['appid_len'] = uid_seq['appid'].map(lambda x:len(x))\n",
    "uid_seq = uid_seq[uid_seq['appid_len']<=300]\n",
    "uid_seq['appid'] = uid_seq['appid'].map(lambda x:\"#\".join(x))\n",
    "\n",
    "del uid_seq['appid_len']\n",
    "\n",
    "all_data = all_data.merge(uid_seq.rename(columns={'appid' : 'usage_seq'}),how='left',on='uid')\n",
    "\n",
    "all_data['usage_seq'] = all_data['usage_seq'].astype('str')\n",
    "key2index = {}\n",
    "app1_list = list(map(split, all_data['usage_seq'].values))\n",
    "app1_length = np.array(list(map(len, app1_list)))\n",
    "app1_key = key2index.copy()\n",
    "\n",
    "max_len_app1 = max(app1_length)\n",
    "app1_list = pad_sequences(app1_list, maxlen=max_len_app1, padding='post', )\n",
    "print(max_len_app1)\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler,StandardScaler\n",
    "from tqdm import tqdm\n",
    "\n",
    "sparse_features = [i for i in all_data.select_dtypes(object).columns if i not in ['uid','age_group']]\n",
    "dense_features = [i for i in all_data.columns if i not in sparse_features+['uid','age_group']]\n",
    "target = ['age_group']\n",
    "for feat in tqdm(sparse_features):\n",
    "    lbl = LabelEncoder()\n",
    "    all_data[feat] = lbl.fit_transform(all_data[feat].astype('str'))\n",
    "\n",
    "mm = StandardScaler()\n",
    "all_data[dense_features] = mm.fit_transform(all_data[dense_features].replace([np.inf,-np.inf],0).fillna(0))\n",
    "\n",
    "choose = all_data['age_group'].notnull()\n",
    "fixlen_feature_columns = [SparseFeat(feat,all_data[feat].nunique()) for feat in sparse_features] + [DenseFeat(feat,1,) for feat in dense_features]\n",
    "varlen_feature_columns = [VarLenSparseFeat('app', len(app_key) + 1, max_len_app, 'mean')] + [VarLenSparseFeat('app1', len(app_key) + 1, max_len_app, 'max')] + \\\n",
    "                         [VarLenSparseFeat('usage_app',len(app1_key) + 1, max_len_app1, 'mean')] + [VarLenSparseFeat('usage_app1', len(app1_key) + 1, max_len_app1,'max')]\n",
    "linear_feature_columns = fixlen_feature_columns + varlen_feature_columns\n",
    "dnn_feature_columns = fixlen_feature_columns + varlen_feature_columns\n",
    "fixlen_feature_names = get_fixlen_feature_names(linear_feature_columns + dnn_feature_columns)\n",
    "varlen_feature_names = get_varlen_feature_names(linear_feature_columns + dnn_feature_columns)\n",
    "len(sparse_features),len(dense_features)\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.keras import backend as K\n",
    "from tensorflow.python.keras.initializers import Zeros, glorot_normal\n",
    "from tensorflow.python.keras.layers import Layer\n",
    "from tensorflow.python.keras.regularizers import l2\n",
    "from deepctr.datas import *\n",
    "from deepctr.contrib import *\n",
    "from deepctr.layers import *\n",
    "from deepctr.models import *\n",
    "from keras.optimizers import Adam\n",
    "from keras.models import Model\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "from keras.layers import concatenate\n",
    "from keras.callbacks import *\n",
    "from keras.constraints import *\n",
    "from keras.layers import *\n",
    "from keras.models import *\n",
    "from keras.initializers import *\n",
    "from keras.optimizers import *\n",
    "\n",
    "label_name = 'age_group'\n",
    "all_data[label_name] = all_data[label_name] - 1\n",
    "all_data[label_name].value_counts()\n",
    "\n",
    "# from tf.keras.activations.softplus\n",
    "\n",
    "def xDeepFM(linear_feature_columns, dnn_feature_columns, embedding_size=8, dnn_hidden_units=(512, 256),\n",
    "            cin_layer_size=(256, 256,), cin_split_half=True, cin_activation='relu', l2_reg_linear=0.00001,\n",
    "            l2_reg_embedding=0.00001, l2_reg_dnn=0, l2_reg_cin=0, init_std=0.0001, seed=2019, dnn_dropout=0,\n",
    "            dnn_activation='relu', dnn_use_bn=True, task='binary'):\n",
    "    \"\"\"Instantiates the xDeepFM architecture.\n",
    "    :param linear_feature_columns: An iterable containing all the features used by linear part of the model.\n",
    "    :param dnn_feature_columns: An iterable containing all the features used by deep part of the model.\n",
    "    :param embedding_size: positive integer,sparse feature embedding_size\n",
    "    :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net\n",
    "    :param cin_layer_size: list,list of positive integer or empty list, the feature maps  in each hidden layer of Compressed Interaction Network\n",
    "    :param cin_split_half: bool.if set to True, half of the feature maps in each hidden will connect to output unit\n",
    "    :param cin_activation: activation function used on feature maps\n",
    "    :param l2_reg_linear: float. L2 regularizer strength applied to linear part\n",
    "    :param l2_reg_embedding: L2 regularizer strength applied to embedding vector\n",
    "    :param l2_reg_dnn: L2 regularizer strength applied to deep net\n",
    "    :param l2_reg_cin: L2 regularizer strength applied to CIN.\n",
    "    :param init_std: float,to use as the initialize std of embedding vector\n",
    "    :param seed: integer ,to use as random seed.\n",
    "    :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.\n",
    "    :param dnn_activation: Activation function to use in DNN\n",
    "    :param dnn_use_bn: bool. Whether use BatchNormalization before activation or not in DNN\n",
    "    :param task: str, ``\"binary\"`` for  binary logloss or  ``\"regression\"`` for regression loss\n",
    "    :return: A Keras model instance.\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "    features = build_data_features(linear_feature_columns + dnn_feature_columns)\n",
    "\n",
    "    datas_list = list(features.values())\n",
    "\n",
    "    sparse_embedding_list, dense_value_list = data_from_feature_columns(features,dnn_feature_columns,\n",
    "                                                                              embedding_size,\n",
    "                                                                              l2_reg_embedding,init_std,\n",
    "                                                                              seed)\n",
    "\n",
    "#     linear_logit \n",
    "    feature_columns = linear_feature_columns\n",
    "    prefix = 'linear'\n",
    "    units = 6\n",
    "    l2_reg = l2_reg_linear\n",
    "    linear_emb_list = [data_from_feature_columns(features,feature_columns,1,l2_reg,init_std,seed,prefix=prefix+str(i))[0] for i in range(units)]\n",
    "    _, dense_data_list = data_from_feature_columns(features,feature_columns,1,l2_reg,init_std,seed,prefix=prefix)\n",
    "    \n",
    "\n",
    "    if len(linear_emb_list[0]) > 1:\n",
    "        linear_term = concat_fun([tf.keras.layers.add(linear_emb) for linear_emb in linear_emb_list])\n",
    "    elif len(linear_emb_list[0]) == 1:\n",
    "        linear_term = concat_fun([linear_emb[0] for linear_emb in linear_emb_list])\n",
    "    else:\n",
    "        linear_term = None\n",
    "    \n",
    "    if len(dense_data_list) > 0:\n",
    "        dense_data__ = dense_data_list[0] if len(\n",
    "            dense_data_list) == 1 else tf.keras.layers.Concatenate()(dense_data_list)\n",
    "        linear_dense_logit = tf.keras.layers.Dense(\n",
    "            units, activation='softplus', use_bias=True, kernel_regularizer=l2(l2_reg))(dense_data__)\n",
    "        \n",
    "        if linear_term is not None:\n",
    "            linear_term = tf.keras.layers.add([linear_dense_logit, linear_term])\n",
    "        else:\n",
    "            linear_term = linear_dense_logit\n",
    "    \n",
    "    linear_logit = tf.keras.layers.Flatten()(linear_term)\n",
    "\n",
    "    fm_data = concat_fun(sparse_embedding_list, axis=1)\n",
    "\n",
    "    if len(cin_layer_size) > 0:\n",
    "        exFM_out = CIN(cin_layer_size, cin_activation,\n",
    "                       cin_split_half, l2_reg_cin, seed)(fm_data)\n",
    "        exFM_logit = tf.keras.layers.Dense(6, activation='softplus', )(exFM_out)\n",
    "        exFM_logit_reg = tf.keras.layers.Dense(1, activation='relu')(exFM_out)\n",
    "\n",
    "    dnn_data_1 = combined_dnn_data(sparse_embedding_list,dense_value_list)\n",
    "    \n",
    "    deep_out_1 = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout,\n",
    "                   dnn_use_bn, seed)(dnn_data_1)\n",
    "    \n",
    "    deep_logit_1 = tf.keras.layers.Dense(\n",
    "        6, use_bias=False, activation='softmax')(deep_out_1)\n",
    "\n",
    "    x = tf.keras.layers.average([exFM_logit,linear_logit,deep_logit_1])\n",
    "    x = tf.keras.layers.concatenate([x,exFM_logit_reg])\n",
    "    x = tf.keras.layers.BatchNormalization()(x)\n",
    "    x = tf.keras.layers.Dense(256)(x)\n",
    "    x = tf.keras.layers.PReLU()(x)\n",
    "    x = tf.keras.layers.BatchNormalization()(x)\n",
    "    output = tf.keras.layers.Dense(6,activation='softmax')(x)\n",
    "    model = tf.keras.models.Model(datas=datas_list, outputs=output)\n",
    "    return model\n",
    "\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = \"3\"\n",
    "\n",
    "from keras.utils import np_utils\n",
    "\n",
    "def make_label(x):\n",
    "    return np_utils.to_categorical(x)\n",
    "\n",
    "def make_data(JB,index):\n",
    "    JB = JB.iloc[index]\n",
    "    fixlen_data = [JB[name].values for name in fixlen_feature_names]\n",
    "    v0 = [app_list[index]]\n",
    "    v1 = [app_list[index]]\n",
    "    v2 = [app1_list[index]]\n",
    "    v3 = [app1_list[index]]\n",
    "    return fixlen_data + v0 + v1 + v2 + v3\n",
    "\n",
    "random_seed = 2019\n",
    "tr_index = choose\n",
    "X_train = all_data[tr_index].reset_index(drop=True)\n",
    "y = all_data[tr_index]['age_group'].reset_index(drop=True).astype(int)\n",
    "X_test = all_data[~tr_index].reset_index(drop=True)\n",
    "print(X_train.shape,X_test.shape)\n",
    "\n",
    "from sklearn.metrics import f1_score,accuracy_score\n",
    "\n",
    "cv_pred = []\n",
    "test_pred = []\n",
    "cv_score = []\n",
    "cv_model = []\n",
    "skf = StratifiedKFold(n_splits=5, random_state=random_seed, shuffle=True)\n",
    "for index, (train_index, test_index) in enumerate(skf.split(X_train, y)):\n",
    "    print(index)\n",
    "    model = xDeepFM(linear_feature_columns, dnn_feature_columns,embedding_size=8,task='multicalss') # xDeepFM DeepFM AFM NFM\n",
    "    model.compile(RAdam(lr=0.01),'categorical_crossentropy',\n",
    "                 metrics = ['accuracy',],)\n",
    "    train_x, test_x, train_y, test_y = X_train.iloc[train_index], X_train.iloc[test_index], y.iloc[train_index], y.iloc[test_index]\n",
    "    \n",
    "    train_x = make_data(X_train,train_index)\n",
    "    train_y = make_label(train_y)\n",
    "    test_x = make_data(X_train,test_index)\n",
    "    test_y = make_label(test_y)\n",
    "    test_data = make_data(all_data,range(2010000,2512500))\n",
    "    history = model.fit(train_x,train_y,batch_size=512,epochs=1,verbose=1,validation_data=(test_x,test_y))\n",
    "    cv_model.append(model)\n",
    "    y_test = model.predict(test_data,batch_size=512)\n",
    "    y_val = model.predict(test_x,batch_size=512)\n",
    "    cv_score.append(accuracy_score(y.iloc[test_index],np.argmax(y_val,axis=1)))\n",
    "    print(cv_score)\n",
    "    cv_pred.append(y_val)\n",
    "    test_pred.append(y_test)\n",
    "\n",
    "cv_pred = np.zeros((X_train.shape[0],6))\n",
    "test_pred = np.zeros((X_test.shape[0],6))\n",
    "for index, (train_index, test_index) in enumerate(skf.split(X_train, y)):\n",
    "    print(index)\n",
    "    train_x, test_x, train_y, test_y = X_train.iloc[train_index], X_train.iloc[test_index], y.iloc[train_index], y.iloc[test_index]\n",
    "    train_x = make_data(X_train,train_index)\n",
    "    train_y = make_label(train_y)\n",
    "    test_x = make_data(X_train,test_index)\n",
    "    test_y = make_label(test_y)\n",
    "    test_data = make_data(all_data,range(2010000,2512500))\n",
    "    y_val = cv_model[index].predict(test_x,batch_size=256,verbose=1)\n",
    "    print(y_val.shape)\n",
    "    cv_pred[test_index] = y_val\n",
    "    test_pred += cv_model[index].predict(test_data,batch_size=256,verbose=1) / 5\n",
    "\n",
    "oof_train = pd.DataFrame(cv_pred)\n",
    "oof_train.columns = ['proba_{}'.format(i) for i in range(6)]\n",
    "oof_train['uid'] = train['uid']\n",
    "\n",
    "oof_test = pd.DataFrame(test_pred)\n",
    "oof_test.columns = ['proba_{}'.format(i) for i in range(6)]\n",
    "oof_test['uid'] = test['uid']\n",
    "\n",
    "oof_train.to_hdf(\"xDeepFM_cv_6449.hdf\",\"train\")\n",
    "oof_test.to_hdf(\"xDeepFM_cv_6449.hdf\",\"test\")\n",
    "# 5855 5993"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
